package bundle2name

import org.slf4j.LoggerFactory
import us.codecraft.webmagic.Page
import us.codecraft.webmagic.Site
import us.codecraft.webmagic.Spider
import us.codecraft.webmagic.processor.PageProcessor

val urls = arrayOf("http://www.wandoujia.com/category/6001","http://www.wandoujia.com/category/6003",
        "http://www.wandoujia.com/category/6004","http://www.wandoujia.com/category/6008",
        "http://www.wandoujia.com/category/6002","http://www.wandoujia.com/category/6007",
        "http://www.wandoujia.com/category/6009","http://www.wandoujia.com/category/6005",
        "http://www.wandoujia.com/category/6006","http://www.wandoujia.com/category/5015",
        "http://www.wandoujia.com/category/5029","http://www.wandoujia.com/category/5018",
        "http://www.wandoujia.com/category/5014","http://www.wandoujia.com/category/5024",
        "http://www.wandoujia.com/category/5019","http://www.wandoujia.com/category/5016",
        "http://www.wandoujia.com/category/5026","http://www.wandoujia.com/category/5017",
        "http://www.wandoujia.com/category/5023","http://www.wandoujia.com/category/5020",
        "http://www.wandoujia.com/category/5021","http://www.wandoujia.com/category/5028",
        "http://www.wandoujia.com/category/5022","http://www.wandoujia.com/category/5027")
fun main(args: Array<String>) {
    val sp = Spider.create(WandoujiaPageProcess())
            .addUrl(*urls)
            .addPipeline(FilePipeline())
            .thread(8)
    sp.run()
}

class WandoujiaPageProcess : PageProcessor {
    val Log = LoggerFactory.getLogger(javaClass)
    override fun getSite(): Site {
        return Site.me().setRetryTimes(3).setSleepTime(1000).setCycleRetryTimes(3).setTimeOut(300_000).setRetrySleepTime(10_000)
    }

    override fun process(page: Page) {
        Log.info(page.url.get())
        val main = page.html.`$`("#j-tag-list li")
        if(main != null){
            val nodes = main.nodes()
            val ps = mutableListOf<BundleName>()
            nodes.forEach { li ->
                val bundle = li.xpath("/*/@data-pn")
                val name = li.xpath("//div[@class='app-desc']/h2[@class='app-title-h2']/a/text()")
                if(name.get() != null)
                    ps.add(BundleName(bundle.get(),name.get()))

            }
            page.putField("ps",ps)
            val tabs = page.html.xpath("//ul[@class='switch-tab cate-tab']/li/a")
            tabs?.let {
                page.addTargetRequests(it.links().all())
            }
            val next = page.html.xpath("//a[@class='page-item next-page ']/@href").get()
            page.addTargetRequest(next)

        }
    }

}